{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-06-09T10:52:05.436738Z",
     "start_time": "2025-06-09T10:52:03.779427Z"
    }
   },
   "source": [
    "# 导包\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from scipy.stats import kurtosis\n",
    "import time\n",
    "import warnings\n",
    "import gc\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "pd.set_option('display.max_columns', None)"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:52:08.775977Z",
     "start_time": "2025-06-09T10:52:08.771918Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\"\"\"\n",
    "    优化DataFrame的数据类型以减少内存占用\n",
    "    将int64转换为int32，float64转换为float32\n",
    "\"\"\"\n",
    "# 定义数据类型优化函数\n",
    "def com(df):\n",
    "\n",
    "    # 找出int64类型的列\n",
    "    l=df.keys()[df.dtypes=='int64']\n",
    "    for i in l:\n",
    "        df[i]=df[i].astype('int32')\n",
    "     # 找出float64类型的列\n",
    "    l=df.keys()[df.dtypes=='float64']\n",
    "    for i in l:\n",
    "        df[i]=df[i].astype('float32')\n",
    "    # 触发垃圾回收\n",
    "    gc.collect()\n",
    "    # 打印\n",
    "    print(df.info())\n",
    "    return df   \n",
    "\n",
    "# 记录开始时间\n",
    "t = time.time()"
   ],
   "id": "d0615ca82bc09c0d",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "##  处理训练数据",
   "id": "624ac3e0e5fe0669"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:52:18.499975Z",
     "start_time": "2025-06-09T10:52:11.201191Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 读取训练数据并处理日期字段\n",
    "train_df = pd.read_csv('data/train.csv', parse_dates=['auditing_date', 'due_date', 'repay_date'])\n",
    "train_df['repay_date'] = train_df[['due_date', 'repay_date']].apply(\n",
    "    # 处理repay_date字段，如果为'\\N'则使用due_date\n",
    "    lambda x: x['repay_date'] if x['repay_date'] != '\\\\N' else x['due_date'], axis=1\n",
    ")\n",
    "# 处理repay_amt字段，如果为'\\N'则设为0\n",
    "train_df['repay_amt'] = train_df['repay_amt'].apply(lambda x: x if x != '\\\\N' else 0).astype('float32')"
   ],
   "id": "a93c8d737f6cab5",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:52:53.998116Z",
     "start_time": "2025-06-09T10:52:19.576782Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 读取新列表数据并合并\n",
    "newlist = pd.read_csv('result_data/newlist.csv', parse_dates=['auditing_date', 'due_date', 'repay_date'])\n",
    "newlist = newlist.merge(train_df[['user_id', 'listing_id', 'auditing_date', 'due_date', 'due_amt']],\n",
    "                        on=['user_id', 'listing_id', 'auditing_date', 'due_date'], how='left')\n",
    "# 转换repay_date为datetime格式\n",
    "train_df['repay_date'] = pd.to_datetime(train_df['repay_date'])\n",
    "# 合并还款金额数据\n",
    "newlist = newlist.merge(train_df[['user_id', 'listing_id', 'auditing_date', 'due_date', 'repay_date', 'repay_amt']],\n",
    "on=['user_id', 'listing_id', 'auditing_date', 'due_date', 'repay_date'], how='left')\n",
    "# 输出\n",
    "print(newlist.repay_amt.sum())"
   ],
   "id": "7c01f213e6cab872",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "396539700.0\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:00.855684Z",
     "start_time": "2025-06-09T10:53:00.193422Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 计算还款天数差\n",
    "newlist['l1'] = (newlist['due_date'] - newlist['repay_date']).dt.days"
   ],
   "id": "10267827806e41f7",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:06.031285Z",
     "start_time": "2025-06-09T10:53:02.966737Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 处理还款金额为0的记录\n",
    "g = newlist.loc[newlist['repay_amt'] == 0]\n",
    "g = g[['user_id', 'listing_id']]\n",
    "g['t'] = 1\n",
    "newlist = newlist.merge(g, on=['user_id', 'listing_id'], how='left')"
   ],
   "id": "4019a9666c9d5e78",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:07.273534Z",
     "start_time": "2025-06-09T10:53:06.801144Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 对于逾期记录，还款金额设为应还金额\n",
    "newlist.loc[(newlist['l1'] < 0) & (newlist['t'] == 1), 'repay_amt'] = newlist.loc[\n",
    "    (newlist['l1'] < 0) & (newlist['t'] == 1), 'due_amt']\n",
    "# 打印处理后的还款金额总和\n",
    "print(newlist.repay_amt.sum())  "
   ],
   "id": "4bddf5d6982341d",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "452026625.2153845\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:09.125501Z",
     "start_time": "2025-06-09T10:53:08.916331Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 填充缺失值为0并删除临时列\n",
    "newlist['repay_amt'] = newlist['repay_amt'].fillna(0)\n",
    "newlist.pop('t')\n",
    "newlist.pop('l1')\n",
    "# 更新训练数据\n",
    "train_df = newlist  \n",
    "# 删除临时变量\n",
    "del newlist  \n",
    "# 垃圾回收\n",
    "gc.collect()  "
   ],
   "id": "2ff2a0d6b4695939",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 准备标签数据",
   "id": "4bff0aa80e12a5a3"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:10.735037Z",
     "start_time": "2025-06-09T10:53:10.599938Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 还款金额标签\n",
    "amt_labels = train_df['repay_amt'].values  \n",
    "# 是否还款标签(二分类)\n",
    "clf_labels = (train_df['repay_amt'] > 0).astype(int).values  \n",
    "# 应还金额\n",
    "train_due_amt_df = train_df[['due_amt']]  \n",
    "# 训练样本数量\n",
    "train_num = train_df.shape[0]  \n",
    "# 删除已处理的列\n",
    "del train_df['repay_amt']  "
   ],
   "id": "f7ffb9f40a749aca",
   "outputs": [],
   "execution_count": 10
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 处理测试数据",
   "id": "cab3673a53a22b9c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:13.935197Z",
     "start_time": "2025-06-09T10:53:12.575382Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 读取测试数据并处理日期字段\n",
    "test_df = pd.read_csv('data/test.csv', parse_dates=['auditing_date', 'due_date'])\n",
    "sub_example = pd.read_csv('data/submission.csv', parse_dates=['repay_date'])\n",
    "tc = test_df.copy()"
   ],
   "id": "31446dd85eb53e28",
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:15.352824Z",
     "start_time": "2025-06-09T10:53:15.083612Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并提交样本中的还款日期\n",
    "test_df = test_df.merge(sub_example[['listing_id', 'repay_date']], on=['listing_id'], how='right')"
   ],
   "id": "19fbdfd583086243",
   "outputs": [],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:16.024244Z",
     "start_time": "2025-06-09T10:53:15.985071Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 添加逾期一天的记录\n",
    "tc['repay_date'] = tc['due_date'] + np.timedelta64(1, 'D')\n",
    "# 合并原始测试数据和添加的逾期记录\n",
    "test_df = pd.concat([test_df, tc], axis=0, ignore_index=True)"
   ],
   "id": "d58e47eff77e2f7a",
   "outputs": [],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:17.914483Z",
     "start_time": "2025-06-09T10:53:17.686973Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并训练和测试数据\n",
    "df = pd.concat([train_df, test_df], axis=0, ignore_index=True)"
   ],
   "id": "996d89e68df4b23",
   "outputs": [],
   "execution_count": 14
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 计算日期相关特征",
   "id": "974c19b93a74aa96"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:21.448667Z",
     "start_time": "2025-06-09T10:53:19.010373Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 距离最后还款日的天数\n",
    "df['l1'] = (df['due_date'] - df['repay_date']).dt.days  \n",
    "# 距离成交日的天数\n",
    "df['l2'] = (df['repay_date'] - df['auditing_date']).dt.days  \n",
    "# 逾期处理\n",
    "df.loc[df['l1'] < 0, 'l2'] = 32  \n",
    "# 总天数\n",
    "df['adays'] = (df['due_date'] - df['auditing_date']).dt.days  "
   ],
   "id": "994390ae3003bf71",
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 处理节假日数据",
   "id": "4749ae7063f28c88"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:26.258497Z",
     "start_time": "2025-06-09T10:53:22.159360Z"
    }
   },
   "cell_type": "code",
   "source": [
    "gx = df[['user_id', 'listing_id', 'repay_date']].copy()\n",
    "# 读取节假日数据\n",
    "hdays = pd.read_table('data/holidays_cn.txt', parse_dates=['date'])  \n",
    "# 是否为节假日\n",
    "hdays['xiu'] = hdays['holiday'] != 'no'  \n",
    "hdays['repay_date'] = hdays['date']\n",
    "# 合并节假日标记\n",
    "df = df.merge(hdays[['repay_date', 'xiu']], on='repay_date', how='left')"
   ],
   "id": "d39d8f0610750d1",
   "outputs": [],
   "execution_count": 16
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 筛选数据集",
   "id": "cbefdc58be8cf032"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:27.701454Z",
     "start_time": "2025-06-09T10:53:27.697960Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 筛选数据集，只使用后1/40的数据(可能是为了快速实验)\n",
    "df = df.iloc[train_num - 1 * (train_num // 40):]\n",
    "clf_labels = clf_labels[train_num - 1 * (train_num // 40):]\n",
    "train_due_amt_df = train_due_amt_df.iloc[train_num - 1 * (train_num // 40):]\n",
    "amt_labels = amt_labels[train_num - 1 * (train_num // 40):]\n",
    "train_num = train_num // 40  # 更新训练样本数量"
   ],
   "id": "4080c566de7fb6db",
   "outputs": [],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T12:29:06.361572Z",
     "start_time": "2025-06-09T12:29:05.632204Z"
    }
   },
   "cell_type": "code",
   "source": "train_due_amt_df.to_csv('result_data/train_due_amt.csv', index=False)",
   "id": "590e8dbb0c8882ea",
   "outputs": [],
   "execution_count": 88
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:10:02.854979Z",
     "start_time": "2025-06-09T11:10:02.842214Z"
    }
   },
   "cell_type": "code",
   "source": [
    "amt_labels_df = pd.DataFrame({'array': amt_labels})\n",
    "amt_labels_df"
   ],
   "id": "479a4b01eb1bb652",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "             array\n",
       "0         0.000000\n",
       "1         0.000000\n",
       "2         0.000000\n",
       "3         0.000000\n",
       "4         0.000000\n",
       "...            ...\n",
       "811313    0.000000\n",
       "811314    0.000000\n",
       "811315    0.000000\n",
       "811316  180.969498\n",
       "811317    0.000000\n",
       "\n",
       "[811318 rows x 1 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>array</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811313</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811314</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811315</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811316</th>\n",
       "      <td>180.969498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811317</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>811318 rows × 1 columns</p>\n",
       "</div>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 74
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:10:28.222178Z",
     "start_time": "2025-06-09T11:10:27.878012Z"
    }
   },
   "cell_type": "code",
   "source": "amt_labels_df.to_csv('result_data/amt_labels.csv', index=False)",
   "id": "c31869883e968929",
   "outputs": [],
   "execution_count": 75
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:29:00.527957Z",
     "start_time": "2025-06-09T11:29:00.523740Z"
    }
   },
   "cell_type": "code",
   "source": "clf_labels",
   "id": "e5e395e56ec07b6",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 1, 0])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 84
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:27:25.377720Z",
     "start_time": "2025-06-09T11:27:25.366381Z"
    }
   },
   "cell_type": "code",
   "source": [
    "clf_labels_df = pd.DataFrame({'array': clf_labels})\n",
    "clf_labels_df"
   ],
   "id": "40a3b8128ef9e1c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "        array\n",
       "0           0\n",
       "1           0\n",
       "2           0\n",
       "3           0\n",
       "4           0\n",
       "...       ...\n",
       "811313      0\n",
       "811314      0\n",
       "811315      0\n",
       "811316      1\n",
       "811317      0\n",
       "\n",
       "[811318 rows x 1 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>array</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811313</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811314</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811315</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811316</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811317</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>811318 rows × 1 columns</p>\n",
       "</div>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 82
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:27:38.058282Z",
     "start_time": "2025-06-09T11:27:37.854775Z"
    }
   },
   "cell_type": "code",
   "source": "clf_labels_df.to_csv('result_data/clf_labels.csv', index=False)",
   "id": "c363374f724a59ff",
   "outputs": [],
   "execution_count": 83
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:29.425825Z",
     "start_time": "2025-06-09T10:53:29.313282Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 准备还款日期数据\n",
    "gx = df[['user_id', 'listing_id', 'repay_date']].copy()"
   ],
   "id": "8f0317f1b982877c",
   "outputs": [],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:36.669419Z",
     "start_time": "2025-06-09T10:53:30.467211Z"
    }
   },
   "cell_type": "code",
   "source": "gx.to_csv('result_data/gx.csv', index=False)",
   "id": "d1a90fa2ab947fcd",
   "outputs": [],
   "execution_count": 19
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:38.041204Z",
     "start_time": "2025-06-09T10:53:37.710716Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 添加日期相关特征\n",
    "i = 0\n",
    "# 当天星期几(0-6)\n",
    "df['last' + str(i) + 'dayofweek'] = (df['repay_date']).dt.dayofweek  \n",
    "# 当天几号(1-31)\n",
    "df['last' + str(i) + 'dayofmonth'] = (df['repay_date']).dt.day  \n",
    "# 当天第几周(0-4)\n",
    "df['last' + str(i) + 'weekofmonth'] = (df['repay_date']).dt.day // 7  \n",
    "# 是否最后一周还款\n",
    "df['im'] = df['l1'] < 7  \n",
    "# 第几周还款\n",
    "df['eweek'] = df['l2'] // 7  "
   ],
   "id": "a412712c55b55d7e",
   "outputs": [],
   "execution_count": 20
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:39.599016Z",
     "start_time": "2025-06-09T10:53:39.594967Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 保存日期特征列名\n",
    "xz = []\n",
    "xz.append('last' + str(i) + 'dayofweek')\n",
    "xz.append('last' + str(i) + 'dayofmonth')\n",
    "xz.append('last' + str(i) + 'weekofmonth')"
   ],
   "id": "78074e3e36977413",
   "outputs": [],
   "execution_count": 21
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:45.011951Z",
     "start_time": "2025-06-09T10:53:41.265272Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并listing信息数据\n",
    "listing_info_df = pd.read_csv('data/listing_info.csv', parse_dates=['auditing_date'])\n",
    "# 计算超额支付\n",
    "listing_info_df['overpay'] = listing_info_df['rate'] * listing_info_df['principal']  \n",
    "# 删除不需要的列\n",
    "del listing_info_df['user_id'], listing_info_df['auditing_date']\n",
    "# 合并到主数据\n",
    "df = df.merge(listing_info_df, on='listing_id', how='left')\n",
    "# 进度标记\n",
    "print(1)  "
   ],
   "id": "6da1feb07e28e712",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "execution_count": 22
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:46.752637Z",
     "start_time": "2025-06-09T10:53:46.748705Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义年龄分段函数\n",
    "def get_age_bin(age):\n",
    "    \"\"\"将年龄分段\"\"\"\n",
    "    if age <= 18:\n",
    "        return 'age<=18'\n",
    "    elif age <= 22:\n",
    "        return '18<age<=22'\n",
    "    elif age <= 26:\n",
    "        return '22<age<=26'\n",
    "    elif age <= 30:\n",
    "        return '26<age<=30'\n",
    "    elif age <= 35:\n",
    "        return '30<age<=35'\n",
    "    elif age <= 40:\n",
    "        return '35<age<=40'\n",
    "    elif age <= 50:\n",
    "        return '40<age<=50'\n",
    "    else:\n",
    "        return '50<age<=90'"
   ],
   "id": "71bc187bcbc4dbe6",
   "outputs": [],
   "execution_count": 23
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:49.523301Z",
     "start_time": "2025-06-09T10:53:48.418365Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并用户信息数据\n",
    "user_info_df = pd.read_csv('data/user_info.csv', parse_dates=['reg_mon', 'insertdate'])\n",
    " # 重命名列\n",
    "user_info_df.rename(columns={'insertdate': 'info_insert_date'}, inplace=True) \n",
    "# 添加年龄分段\n",
    "user_info_df['age_bin'] = user_info_df['age'].apply(lambda age: get_age_bin(age))\n",
    "# 创建城市+年龄分段组合特征\n",
    "user_info_df['city_age'] = user_info_df['id_city'].astype(str) + user_info_df['age_bin'].astype(str)"
   ],
   "id": "9625cf9d351b267f",
   "outputs": [],
   "execution_count": 24
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:51.791334Z",
     "start_time": "2025-06-09T10:53:50.581152Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 计算每个用户的记录数\n",
    "g = user_info_df.groupby('user_id').size().reset_index(name='us')\n",
    "user_info_df = user_info_df.merge(g, on='user_id', how='left')\n",
    "# 添加电话和城市是否匹配的标记\n",
    "user_info_df['uec'] = user_info_df['cell_province'] == user_info_df['id_province']\n",
    "# 按插入日期排序并去重\n",
    "user_info_df = user_info_df.sort_values(by='info_insert_date', ascending=False).drop_duplicates('user_id').reset_index(drop=True)"
   ],
   "id": "de3e318c77036ed",
   "outputs": [],
   "execution_count": 25
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:53.792929Z",
     "start_time": "2025-06-09T10:53:52.430098Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并到主数据\n",
    "df = df.merge(user_info_df, on='user_id', how='left')"
   ],
   "id": "81c2a5866f35909",
   "outputs": [],
   "execution_count": 26
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:53:58.607033Z",
     "start_time": "2025-06-09T10:53:55.894249Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并用户标签数据\n",
    "user_tag_df = pd.read_csv('data/user_taglist.csv', parse_dates=['insertdate'])\n",
    "user_tag_df.rename(columns={'insertdate': 'tag_insert_date'}, inplace=True)"
   ],
   "id": "2cf6236cad44f9dd",
   "outputs": [],
   "execution_count": 27
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:00.049929Z",
     "start_time": "2025-06-09T10:53:59.800824Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 计算每个用户的标签数量\n",
    "g = user_tag_df.groupby('user_id').size().reset_index(name='uts')\n",
    "user_tag_df = user_tag_df.merge(g, on='user_id', how='left')"
   ],
   "id": "78ac9f33874dba50",
   "outputs": [],
   "execution_count": 28
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:01.912152Z",
     "start_time": "2025-06-09T10:54:01.600358Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并每个用户的所有标签\n",
    "g = user_tag_df.groupby('user_id')['taglist'].sum().reset_index(name='taglist')"
   ],
   "id": "6d1fc0862aa081af",
   "outputs": [],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:04.909205Z",
     "start_time": "2025-06-09T10:54:03.402267Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 按插入日期排序并去重\n",
    "user_tag_df = user_tag_df.sort_values(by='tag_insert_date', ascending=False).drop_duplicates('user_id').reset_index(drop=True)\n",
    "user_tag_df.pop('taglist')\n",
    "user_tag_df = user_tag_df.merge(g, on='user_id', how='left')\n",
    "# 合并到主数据\n",
    "df = df.merge(user_tag_df, on='user_id', how='left')\n",
    "# 进度标记\n",
    "print(2)  "
   ],
   "id": "afa07c0d5e96703",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:23.871940Z",
     "start_time": "2025-06-09T10:54:06.027886Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 重新读取listing信息数据\n",
    "listing_info_df = pd.read_csv('data/listing_info.csv', parse_dates=['auditing_date'])\n",
    "listing_info_df['overpay'] = listing_info_df['rate'] * listing_info_df['principal']\n",
    "# 删除不需要的列\n",
    "del listing_info_df['user_id']\n",
    "# 处理用户还款日志数据\n",
    "repay_log_df = pd.read_csv('data/user_repay_logs.csv', parse_dates=['due_date', 'repay_date'])\n",
    "repay_log_df = repay_log_df.merge(listing_info_df, on='listing_id', how='left')\n",
    "# 过滤掉2020年的数据\n",
    "repay_log_df = repay_log_df.loc[repay_log_df['due_date'].dt.year != 2020]"
   ],
   "id": "44cad12588eb1ce2",
   "outputs": [],
   "execution_count": 31
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:26.143721Z",
     "start_time": "2025-06-09T10:54:25.009573Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 为还款日志添加特征\n",
    "repay_log_df['w'] = repay_log_df['repay_date'].dt.dayofweek  # 还款日星期几\n",
    "repay_log_df['early_repay_days'] = (repay_log_df['due_date'] - repay_log_df['repay_date']).dt.days"
   ],
   "id": "bf0509ea83cd2a3c",
   "outputs": [],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:26.933416Z",
     "start_time": "2025-06-09T10:54:26.889833Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 提前还款天数\n",
    "# 金额分段标记\n",
    "repay_log_df['first'] = repay_log_df['due_amt'] > 5000\n",
    "repay_log_df['second'] = (repay_log_df['due_amt'] > 1000) & (repay_log_df['due_amt'] <= 5000)"
   ],
   "id": "34946dddfabef88c",
   "outputs": [],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:28.750922Z",
     "start_time": "2025-06-09T10:54:28.019866Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 计算日期差特征\n",
    "repay_log_df['l1'] = (repay_log_df['due_date'] - repay_log_df['repay_date']).dt.days\n",
    "repay_log_df['l2'] = (repay_log_df['repay_date'] - repay_log_df['auditing_date']).dt.days\n",
    "# 打印最小还款天数差\n",
    "print(repay_log_df['l1'].min())  "
   ],
   "id": "e1f2ce7b838c0e75",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-66654\n"
     ]
    }
   ],
   "execution_count": 34
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:29.627129Z",
     "start_time": "2025-06-09T10:54:29.516895Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 处理逾期记录\n",
    "repay_log_df.loc[repay_log_df['l1'] < 0, 'l1'] = -1\n",
    "repay_log_df.loc[repay_log_df['l1'] < 0, 'l2'] = 32\n",
    "# 打印处理后的最小还款天数差\n",
    "print(repay_log_df['l1'].min())  "
   ],
   "id": "c0466f8ff0430d8c",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1\n"
     ]
    }
   ],
   "execution_count": 35
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:31.166783Z",
     "start_time": "2025-06-09T10:54:30.607312Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 是否最后一周还款\n",
    "repay_log_df['im'] = repay_log_df['l1'] < 7  \n",
    "# 还款周数\n",
    "repay_log_df['eweek'] = repay_log_df['l2'] // 7  \n",
    "# 应还日与审核日天数差\n",
    "repay_log_df['dd'] = (repay_log_df['due_date'] - repay_log_df['auditing_date']).dt.days  \n",
    " # 主数据中添加相同特征\n",
    "df['dd'] = (df['due_date'] - df['auditing_date']).dt.days "
   ],
   "id": "710c819ba4134729",
   "outputs": [],
   "execution_count": 36
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:34.505635Z",
     "start_time": "2025-06-09T10:54:32.126538Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义还款特征列表\n",
    "rpf = ['w', 'first', 'second', 'l1', 'im', 'eweek', 'l2', 'dd']\n",
    "# 过滤提前还款天数<=31的记录\n",
    "r = repay_log_df.loc[(repay_log_df['early_repay_days'] <= 31)].copy()"
   ],
   "id": "2aa5d3acd9c1916f",
   "outputs": [],
   "execution_count": 37
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:54:35.033780Z",
     "start_time": "2025-06-09T10:54:35.019875Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义还款特征生成函数\n",
    "def repayf(df, r, f, p, flag):\n",
    "    \"\"\"\n",
    "    生成还款相关的统计特征\n",
    "    df: 主数据\n",
    "    r: 还款日志数据\n",
    "    f: 分组字段\n",
    "    p: 特征前缀\n",
    "    flag: 标志位，0表示生成基础特征，1表示生成扩展特征\n",
    "    \"\"\"\n",
    "    t = time.time()\n",
    "\n",
    "    if flag == 0:\n",
    "        # 生成基础还款特征\n",
    "        r['due'] = r['early_repay_days'] == 0  # 是否最后一天还款\n",
    "        # 计算最后一天还款的统计量\n",
    "        gr = r.groupby(f)['due'].agg(\n",
    "            **{\n",
    "                f'{f}{p}dsum': 'sum',\n",
    "                f'{f}{p}dsize': 'size'\n",
    "            }\n",
    "        ).reset_index()\n",
    "        # 计算最后一天还款率\n",
    "        gr[f + p + 'drate'] = (0.0000001 + gr[f + p + 'dsum']) / (0.0000001 + gr[f + p + 'dsize'])\n",
    "        gr = gr.loc[gr[f + p + 'dsize'] > 4]  # 过滤样本量太小的组\n",
    "        df = df.merge(gr, on=f, how='left')\n",
    "        \n",
    "        # 计算逾期统计量\n",
    "        r['late'] = r['early_repay_days'] < 0  # 是否逾期\n",
    "        gr = r.groupby(f)['due'].agg(\n",
    "            **{\n",
    "                f'{f}{p}latesum': 'sum',\n",
    "                f'{f}{p}latesize': 'size'\n",
    "            }\n",
    "        ).reset_index()\n",
    "        # 计算逾期率\n",
    "        gr[f + p + 'laterate'] = (0.0000001 + gr[f + p + 'latesum']) / (0.0000001 + gr[f + p + 'latesize'])\n",
    "        gr = gr.loc[gr[f + p + 'latesize'] > 4]  # 过滤样本量太小的组\n",
    "        df = df.merge(gr, on=f, how='left')\n",
    "        \n",
    "        # 计算过早还款统计量\n",
    "        r['tooearly'] = r['early_repay_days'] > 24  # 是否过早还款\n",
    "        gr = r.groupby(f)['due'].agg(\n",
    "            **{\n",
    "                f'{f}{p}tooearlysum': 'sum',\n",
    "                f'{f}{p}tooearlysize': 'size'\n",
    "            }\n",
    "        ).reset_index()\n",
    "        # 计算过早还款率\n",
    "        gr[f + p + 'tooearlyrate'] = (0.0000001 + gr[f + p + 'tooearlysum']) / (0.0000001 + gr[f + p + 'tooearlysize'])\n",
    "        # 过滤样本量太小的组\n",
    "        gr = gr.loc[gr[f + p + 'tooearlysize'] > 4]  \n",
    "        df = df.merge(gr, on=f, how='left')\n",
    "\n",
    "    # 添加日期相关特征\n",
    "    # 还款日\n",
    "    r['day'] = r['repay_date'].dt.day  \n",
    "    # 还款周数\n",
    "    r['wom'] = r['repay_date'].dt.day // 7  \n",
    "    xr = r.copy()\n",
    "    \n",
    "    # 按星期几统计还款行为\n",
    "    g = xr.groupby([f, 'w']).size().reset_index(name='ws')\n",
    "    g1 = r.groupby(f).size().reset_index(name='us')\n",
    "    g = g.merge(g1, on=f, how='right')\n",
    "    g['us'].fillna(0.0001)\n",
    "     # 减去当前记录\n",
    "    g['ws'] = g['ws'] - 1 \n",
    "    # 减去当前记录\n",
    "    g['us'] = g['us'] - 1  \n",
    "    # 计算比率\n",
    "    g['wus'] = (0.000001 + g['ws']) / (0.000001 + g['us'])  \n",
    "    # 过滤样本量太小的组\n",
    "    g = g.loc[g['us'] > 4]  \n",
    "    \n",
    "    # 合并星期几特征\n",
    "    for i in range(1):\n",
    "        gg = g.copy()\n",
    "        gg.pop('us')\n",
    "        gg.columns = [f, 'last' + str(i) + 'dayofmonth', f + p + 'last' + str(i) + 'daymonthtrickws',\n",
    "                      f + p + 'last' + str(i) + 'daymonthtrickwus']\n",
    "        df = df.merge(gg, on=[f, 'last' + str(i) + 'dayofmonth'], how='left')\n",
    "    \n",
    "    # 按还款日统计还款行为\n",
    "    g = xr.groupby([f, 'day']).size().reset_index(name='ws')\n",
    "    g1 = r.groupby(f).size().reset_index(name='us')\n",
    "    g = g.merge(g1, on=f, how='right')\n",
    "    g['us'].fillna(0.0001)\n",
    "    g['ws'] = g['ws'] - 1\n",
    "    g['us'] = g['us'] - 1\n",
    "    g['wus'] = (0.000001 + g['ws']) / (0.000001 + g['us'])\n",
    "    g = g.loc[g['us'] > 4]\n",
    "    \n",
    "    # 合并还款日特征\n",
    "    for i in range(1):\n",
    "        gg = g.copy()\n",
    "        gg.pop('us')\n",
    "        gg.columns = [f, 'last' + str(i) + 'dayofweek', f + p + 'last' + str(i) + 'daytrickws',\n",
    "                      f + p + 'last' + str(i) + 'daytrickwus']\n",
    "        df = df.merge(gg, on=[f, 'last' + str(i) + 'dayofweek'], how='left')\n",
    "    \n",
    "    # 按还款周数统计还款行为\n",
    "    g = xr.groupby([f, 'wom']).size().reset_index(name='ws')\n",
    "    g1 = r.groupby(f).size().reset_index(name='us')\n",
    "    g = g.merge(g1, on=f, how='right')\n",
    "    g['us'].fillna(0.0001)\n",
    "    g['ws'] = g['ws'] - 1\n",
    "    g['us'] = g['us'] - 1\n",
    "    g['wus'] = (0.000001 + g['ws']) / (0.000001 + g['us'])\n",
    "    g = g.loc[g['us'] > 4]\n",
    "    \n",
    "    # 合并还款周数特征\n",
    "    for i in range(1):\n",
    "        gg = g.copy()\n",
    "        gg.pop('us')\n",
    "        gg.columns = [f, 'last' + str(i) + 'weekofmonth', f + p + 'last' + str(i) + 'womtrickws',\n",
    "                      f + p + 'last' + str(i) + 'womtrickwus']\n",
    "        df = df.merge(gg, on=[f, 'last' + str(i) + 'weekofmonth'], how='left')\n",
    "    \n",
    "    print(f)\n",
    "    print('runtime: {}\\n'.format(time.time() - t))\n",
    "    gc.collect()\n",
    "    # 优化数据类型\n",
    "    df = com(df)  \n",
    "    return df"
   ],
   "id": "4a7b8500cb6e8f5b",
   "outputs": [],
   "execution_count": 38
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:55:03.675062Z",
     "start_time": "2025-06-09T10:54:39.604647Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 添加年龄分段\n",
    "df['a'] = df['age'] // 10\n",
    "# 生成用户ID相关的还款特征\n",
    "df = repayf(df, r, 'user_id', '', 0)\n",
    "c = list(df.keys())\n",
    "print('count', c.count('user_idlast0daymonthtrickws'))"
   ],
   "id": "9506fdbac1ffc48e",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_id\n",
      "runtime: 23.320115089416504\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Data columns (total 50 columns):\n",
      " #   Column                        Dtype         \n",
      "---  ------                        -----         \n",
      " 0   user_id                       int32         \n",
      " 1   listing_id                    int32         \n",
      " 2   auditing_date                 datetime64[ns]\n",
      " 3   due_date                      datetime64[ns]\n",
      " 4   repay_date                    datetime64[ns]\n",
      " 5   due_amt                       float32       \n",
      " 6   l1                            int32         \n",
      " 7   l2                            int32         \n",
      " 8   adays                         int32         \n",
      " 9   xiu                           bool          \n",
      " 10  last0dayofweek                int32         \n",
      " 11  last0dayofmonth               int32         \n",
      " 12  last0weekofmonth              int32         \n",
      " 13  im                            bool          \n",
      " 14  eweek                         int32         \n",
      " 15  term                          int32         \n",
      " 16  rate                          float32       \n",
      " 17  principal                     int32         \n",
      " 18  overpay                       float32       \n",
      " 19  reg_mon                       datetime64[ns]\n",
      " 20  gender                        object        \n",
      " 21  age                           int32         \n",
      " 22  cell_province                 object        \n",
      " 23  id_province                   object        \n",
      " 24  id_city                       object        \n",
      " 25  info_insert_date              datetime64[ns]\n",
      " 26  age_bin                       object        \n",
      " 27  city_age                      object        \n",
      " 28  us                            int32         \n",
      " 29  uec                           bool          \n",
      " 30  tag_insert_date               datetime64[ns]\n",
      " 31  uts                           float32       \n",
      " 32  taglist                       object        \n",
      " 33  dd                            int32         \n",
      " 34  a                             int32         \n",
      " 35  user_iddsum                   float32       \n",
      " 36  user_iddsize                  float32       \n",
      " 37  user_iddrate                  float32       \n",
      " 38  user_idlatesum                float32       \n",
      " 39  user_idlatesize               float32       \n",
      " 40  user_idlaterate               float32       \n",
      " 41  user_idtooearlysum            float32       \n",
      " 42  user_idtooearlysize           float32       \n",
      " 43  user_idtooearlyrate           float32       \n",
      " 44  user_idlast0daymonthtrickws   float32       \n",
      " 45  user_idlast0daymonthtrickwus  float32       \n",
      " 46  user_idlast0daytrickws        float32       \n",
      " 47  user_idlast0daytrickwus       float32       \n",
      " 48  user_idlast0womtrickws        float32       \n",
      " 49  user_idlast0womtrickwus       float32       \n",
      "dtypes: bool(3), datetime64[ns](6), float32(19), int32(15), object(7)\n",
      "memory usage: 1.1+ GB\n",
      "None\n",
      "count 1\n"
     ]
    }
   ],
   "execution_count": 39
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:55:24.493908Z",
     "start_time": "2025-06-09T10:55:06.492641Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 过滤还款日志数据\n",
    "a = test_df.auditing_date.min()\n",
    "r = r.loc[\n",
    "    ((r['repay_date'].astype('str') == '2200-01-01') & (r['due_date'] < a)) \n",
    "        |\n",
    "    ((r['repay_date'].astype('str') != '2200-01-01') & (r['repay_date'] < a))\n",
    "]"
   ],
   "id": "591c05199aa4cebf",
   "outputs": [],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:56:03.241177Z",
     "start_time": "2025-06-09T10:55:25.267155Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 生成城市相关的还款特征\n",
    "r = r.merge(user_info_df, on='user_id', how='left')\n",
    "df = repayf(df, r, 'id_city', '', 0)\n",
    "# 生成还款天数相关的特征\n",
    "df = repayf(df, r, 'l2', '', 1)\n",
    "df = repayf(df, r, 'l1', '', 1)"
   ],
   "id": "10b4d6098e322ae",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id_city\n",
      "runtime: 15.06229043006897\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Data columns (total 65 columns):\n",
      " #   Column                        Dtype         \n",
      "---  ------                        -----         \n",
      " 0   user_id                       int32         \n",
      " 1   listing_id                    int32         \n",
      " 2   auditing_date                 datetime64[ns]\n",
      " 3   due_date                      datetime64[ns]\n",
      " 4   repay_date                    datetime64[ns]\n",
      " 5   due_amt                       float32       \n",
      " 6   l1                            int32         \n",
      " 7   l2                            int32         \n",
      " 8   adays                         int32         \n",
      " 9   xiu                           bool          \n",
      " 10  last0dayofweek                int32         \n",
      " 11  last0dayofmonth               int32         \n",
      " 12  last0weekofmonth              int32         \n",
      " 13  im                            bool          \n",
      " 14  eweek                         int32         \n",
      " 15  term                          int32         \n",
      " 16  rate                          float32       \n",
      " 17  principal                     int32         \n",
      " 18  overpay                       float32       \n",
      " 19  reg_mon                       datetime64[ns]\n",
      " 20  gender                        object        \n",
      " 21  age                           int32         \n",
      " 22  cell_province                 object        \n",
      " 23  id_province                   object        \n",
      " 24  id_city                       object        \n",
      " 25  info_insert_date              datetime64[ns]\n",
      " 26  age_bin                       object        \n",
      " 27  city_age                      object        \n",
      " 28  us                            int32         \n",
      " 29  uec                           bool          \n",
      " 30  tag_insert_date               datetime64[ns]\n",
      " 31  uts                           float32       \n",
      " 32  taglist                       object        \n",
      " 33  dd                            int32         \n",
      " 34  a                             int32         \n",
      " 35  user_iddsum                   float32       \n",
      " 36  user_iddsize                  float32       \n",
      " 37  user_iddrate                  float32       \n",
      " 38  user_idlatesum                float32       \n",
      " 39  user_idlatesize               float32       \n",
      " 40  user_idlaterate               float32       \n",
      " 41  user_idtooearlysum            float32       \n",
      " 42  user_idtooearlysize           float32       \n",
      " 43  user_idtooearlyrate           float32       \n",
      " 44  user_idlast0daymonthtrickws   float32       \n",
      " 45  user_idlast0daymonthtrickwus  float32       \n",
      " 46  user_idlast0daytrickws        float32       \n",
      " 47  user_idlast0daytrickwus       float32       \n",
      " 48  user_idlast0womtrickws        float32       \n",
      " 49  user_idlast0womtrickwus       float32       \n",
      " 50  id_citydsum                   float32       \n",
      " 51  id_citydsize                  float32       \n",
      " 52  id_citydrate                  float32       \n",
      " 53  id_citylatesum                float32       \n",
      " 54  id_citylatesize               float32       \n",
      " 55  id_citylaterate               float32       \n",
      " 56  id_citytooearlysum            float32       \n",
      " 57  id_citytooearlysize           float32       \n",
      " 58  id_citytooearlyrate           float32       \n",
      " 59  id_citylast0daymonthtrickws   float32       \n",
      " 60  id_citylast0daymonthtrickwus  float32       \n",
      " 61  id_citylast0daytrickws        float32       \n",
      " 62  id_citylast0daytrickwus       float32       \n",
      " 63  id_citylast0womtrickws        float32       \n",
      " 64  id_citylast0womtrickwus       float32       \n",
      "dtypes: bool(3), datetime64[ns](6), float32(34), int32(15), object(7)\n",
      "memory usage: 1.4+ GB\n",
      "None\n",
      "l2\n",
      "runtime: 7.518704652786255\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Data columns (total 71 columns):\n",
      " #   Column                        Dtype         \n",
      "---  ------                        -----         \n",
      " 0   user_id                       int32         \n",
      " 1   listing_id                    int32         \n",
      " 2   auditing_date                 datetime64[ns]\n",
      " 3   due_date                      datetime64[ns]\n",
      " 4   repay_date                    datetime64[ns]\n",
      " 5   due_amt                       float32       \n",
      " 6   l1                            int32         \n",
      " 7   l2                            int32         \n",
      " 8   adays                         int32         \n",
      " 9   xiu                           bool          \n",
      " 10  last0dayofweek                int32         \n",
      " 11  last0dayofmonth               int32         \n",
      " 12  last0weekofmonth              int32         \n",
      " 13  im                            bool          \n",
      " 14  eweek                         int32         \n",
      " 15  term                          int32         \n",
      " 16  rate                          float32       \n",
      " 17  principal                     int32         \n",
      " 18  overpay                       float32       \n",
      " 19  reg_mon                       datetime64[ns]\n",
      " 20  gender                        object        \n",
      " 21  age                           int32         \n",
      " 22  cell_province                 object        \n",
      " 23  id_province                   object        \n",
      " 24  id_city                       object        \n",
      " 25  info_insert_date              datetime64[ns]\n",
      " 26  age_bin                       object        \n",
      " 27  city_age                      object        \n",
      " 28  us                            int32         \n",
      " 29  uec                           bool          \n",
      " 30  tag_insert_date               datetime64[ns]\n",
      " 31  uts                           float32       \n",
      " 32  taglist                       object        \n",
      " 33  dd                            int32         \n",
      " 34  a                             int32         \n",
      " 35  user_iddsum                   float32       \n",
      " 36  user_iddsize                  float32       \n",
      " 37  user_iddrate                  float32       \n",
      " 38  user_idlatesum                float32       \n",
      " 39  user_idlatesize               float32       \n",
      " 40  user_idlaterate               float32       \n",
      " 41  user_idtooearlysum            float32       \n",
      " 42  user_idtooearlysize           float32       \n",
      " 43  user_idtooearlyrate           float32       \n",
      " 44  user_idlast0daymonthtrickws   float32       \n",
      " 45  user_idlast0daymonthtrickwus  float32       \n",
      " 46  user_idlast0daytrickws        float32       \n",
      " 47  user_idlast0daytrickwus       float32       \n",
      " 48  user_idlast0womtrickws        float32       \n",
      " 49  user_idlast0womtrickwus       float32       \n",
      " 50  id_citydsum                   float32       \n",
      " 51  id_citydsize                  float32       \n",
      " 52  id_citydrate                  float32       \n",
      " 53  id_citylatesum                float32       \n",
      " 54  id_citylatesize               float32       \n",
      " 55  id_citylaterate               float32       \n",
      " 56  id_citytooearlysum            float32       \n",
      " 57  id_citytooearlysize           float32       \n",
      " 58  id_citytooearlyrate           float32       \n",
      " 59  id_citylast0daymonthtrickws   float32       \n",
      " 60  id_citylast0daymonthtrickwus  float32       \n",
      " 61  id_citylast0daytrickws        float32       \n",
      " 62  id_citylast0daytrickwus       float32       \n",
      " 63  id_citylast0womtrickws        float32       \n",
      " 64  id_citylast0womtrickwus       float32       \n",
      " 65  l2last0daymonthtrickws        float32       \n",
      " 66  l2last0daymonthtrickwus       float32       \n",
      " 67  l2last0daytrickws             float32       \n",
      " 68  l2last0daytrickwus            float32       \n",
      " 69  l2last0womtrickws             int32         \n",
      " 70  l2last0womtrickwus            float32       \n",
      "dtypes: bool(3), datetime64[ns](6), float32(39), int32(16), object(7)\n",
      "memory usage: 1.5+ GB\n",
      "None\n",
      "l1\n",
      "runtime: 8.908080101013184\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Data columns (total 77 columns):\n",
      " #   Column                        Dtype         \n",
      "---  ------                        -----         \n",
      " 0   user_id                       int32         \n",
      " 1   listing_id                    int32         \n",
      " 2   auditing_date                 datetime64[ns]\n",
      " 3   due_date                      datetime64[ns]\n",
      " 4   repay_date                    datetime64[ns]\n",
      " 5   due_amt                       float32       \n",
      " 6   l1                            int32         \n",
      " 7   l2                            int32         \n",
      " 8   adays                         int32         \n",
      " 9   xiu                           bool          \n",
      " 10  last0dayofweek                int32         \n",
      " 11  last0dayofmonth               int32         \n",
      " 12  last0weekofmonth              int32         \n",
      " 13  im                            bool          \n",
      " 14  eweek                         int32         \n",
      " 15  term                          int32         \n",
      " 16  rate                          float32       \n",
      " 17  principal                     int32         \n",
      " 18  overpay                       float32       \n",
      " 19  reg_mon                       datetime64[ns]\n",
      " 20  gender                        object        \n",
      " 21  age                           int32         \n",
      " 22  cell_province                 object        \n",
      " 23  id_province                   object        \n",
      " 24  id_city                       object        \n",
      " 25  info_insert_date              datetime64[ns]\n",
      " 26  age_bin                       object        \n",
      " 27  city_age                      object        \n",
      " 28  us                            int32         \n",
      " 29  uec                           bool          \n",
      " 30  tag_insert_date               datetime64[ns]\n",
      " 31  uts                           float32       \n",
      " 32  taglist                       object        \n",
      " 33  dd                            int32         \n",
      " 34  a                             int32         \n",
      " 35  user_iddsum                   float32       \n",
      " 36  user_iddsize                  float32       \n",
      " 37  user_iddrate                  float32       \n",
      " 38  user_idlatesum                float32       \n",
      " 39  user_idlatesize               float32       \n",
      " 40  user_idlaterate               float32       \n",
      " 41  user_idtooearlysum            float32       \n",
      " 42  user_idtooearlysize           float32       \n",
      " 43  user_idtooearlyrate           float32       \n",
      " 44  user_idlast0daymonthtrickws   float32       \n",
      " 45  user_idlast0daymonthtrickwus  float32       \n",
      " 46  user_idlast0daytrickws        float32       \n",
      " 47  user_idlast0daytrickwus       float32       \n",
      " 48  user_idlast0womtrickws        float32       \n",
      " 49  user_idlast0womtrickwus       float32       \n",
      " 50  id_citydsum                   float32       \n",
      " 51  id_citydsize                  float32       \n",
      " 52  id_citydrate                  float32       \n",
      " 53  id_citylatesum                float32       \n",
      " 54  id_citylatesize               float32       \n",
      " 55  id_citylaterate               float32       \n",
      " 56  id_citytooearlysum            float32       \n",
      " 57  id_citytooearlysize           float32       \n",
      " 58  id_citytooearlyrate           float32       \n",
      " 59  id_citylast0daymonthtrickws   float32       \n",
      " 60  id_citylast0daymonthtrickwus  float32       \n",
      " 61  id_citylast0daytrickws        float32       \n",
      " 62  id_citylast0daytrickwus       float32       \n",
      " 63  id_citylast0womtrickws        float32       \n",
      " 64  id_citylast0womtrickwus       float32       \n",
      " 65  l2last0daymonthtrickws        float32       \n",
      " 66  l2last0daymonthtrickwus       float32       \n",
      " 67  l2last0daytrickws             float32       \n",
      " 68  l2last0daytrickwus            float32       \n",
      " 69  l2last0womtrickws             int32         \n",
      " 70  l2last0womtrickwus            float32       \n",
      " 71  l1last0daymonthtrickws        float32       \n",
      " 72  l1last0daymonthtrickwus       float32       \n",
      " 73  l1last0daytrickws             float32       \n",
      " 74  l1last0daytrickwus            float32       \n",
      " 75  l1last0womtrickws             float32       \n",
      " 76  l1last0womtrickwus            float32       \n",
      "dtypes: bool(3), datetime64[ns](6), float32(45), int32(16), object(7)\n",
      "memory usage: 1.6+ GB\n",
      "None\n"
     ]
    }
   ],
   "execution_count": 41
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:56:05.614666Z",
     "start_time": "2025-06-09T10:56:03.243667Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 处理第一期还款记录\n",
    "repay_log_df = repay_log_df[repay_log_df['order_id'] == 1].reset_index(drop=True)\n",
    "r = repay_log_df.loc[(repay_log_df['early_repay_days'] <= 31)].copy()"
   ],
   "id": "b91761c37a03b34a",
   "outputs": [],
   "execution_count": 42
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:56:26.726703Z",
     "start_time": "2025-06-09T10:56:09.313780Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 生成第一期还款相关特征\n",
    "df = repayf(df, r, 'user_id', 'j', 0)\n",
    "r = r.loc[\n",
    "    ((r['repay_date'].astype('str') == '2200-01-01') & (r['due_date'] < a)) \n",
    "        | \n",
    "    ((r['repay_date'].astype('str') != '2200-01-01') & (r['repay_date'] < a))\n",
    "    ]\n",
    "r = r.merge(user_info_df, on='user_id', how='left')\n",
    "r['a'] = r['age'] // 10"
   ],
   "id": "a167ada430a25462",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_id\n",
      "runtime: 12.409613370895386\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Data columns (total 92 columns):\n",
      " #   Column                         Dtype         \n",
      "---  ------                         -----         \n",
      " 0   user_id                        int32         \n",
      " 1   listing_id                     int32         \n",
      " 2   auditing_date                  datetime64[ns]\n",
      " 3   due_date                       datetime64[ns]\n",
      " 4   repay_date                     datetime64[ns]\n",
      " 5   due_amt                        float32       \n",
      " 6   l1                             int32         \n",
      " 7   l2                             int32         \n",
      " 8   adays                          int32         \n",
      " 9   xiu                            bool          \n",
      " 10  last0dayofweek                 int32         \n",
      " 11  last0dayofmonth                int32         \n",
      " 12  last0weekofmonth               int32         \n",
      " 13  im                             bool          \n",
      " 14  eweek                          int32         \n",
      " 15  term                           int32         \n",
      " 16  rate                           float32       \n",
      " 17  principal                      int32         \n",
      " 18  overpay                        float32       \n",
      " 19  reg_mon                        datetime64[ns]\n",
      " 20  gender                         object        \n",
      " 21  age                            int32         \n",
      " 22  cell_province                  object        \n",
      " 23  id_province                    object        \n",
      " 24  id_city                        object        \n",
      " 25  info_insert_date               datetime64[ns]\n",
      " 26  age_bin                        object        \n",
      " 27  city_age                       object        \n",
      " 28  us                             int32         \n",
      " 29  uec                            bool          \n",
      " 30  tag_insert_date                datetime64[ns]\n",
      " 31  uts                            float32       \n",
      " 32  taglist                        object        \n",
      " 33  dd                             int32         \n",
      " 34  a                              int32         \n",
      " 35  user_iddsum                    float32       \n",
      " 36  user_iddsize                   float32       \n",
      " 37  user_iddrate                   float32       \n",
      " 38  user_idlatesum                 float32       \n",
      " 39  user_idlatesize                float32       \n",
      " 40  user_idlaterate                float32       \n",
      " 41  user_idtooearlysum             float32       \n",
      " 42  user_idtooearlysize            float32       \n",
      " 43  user_idtooearlyrate            float32       \n",
      " 44  user_idlast0daymonthtrickws    float32       \n",
      " 45  user_idlast0daymonthtrickwus   float32       \n",
      " 46  user_idlast0daytrickws         float32       \n",
      " 47  user_idlast0daytrickwus        float32       \n",
      " 48  user_idlast0womtrickws         float32       \n",
      " 49  user_idlast0womtrickwus        float32       \n",
      " 50  id_citydsum                    float32       \n",
      " 51  id_citydsize                   float32       \n",
      " 52  id_citydrate                   float32       \n",
      " 53  id_citylatesum                 float32       \n",
      " 54  id_citylatesize                float32       \n",
      " 55  id_citylaterate                float32       \n",
      " 56  id_citytooearlysum             float32       \n",
      " 57  id_citytooearlysize            float32       \n",
      " 58  id_citytooearlyrate            float32       \n",
      " 59  id_citylast0daymonthtrickws    float32       \n",
      " 60  id_citylast0daymonthtrickwus   float32       \n",
      " 61  id_citylast0daytrickws         float32       \n",
      " 62  id_citylast0daytrickwus        float32       \n",
      " 63  id_citylast0womtrickws         float32       \n",
      " 64  id_citylast0womtrickwus        float32       \n",
      " 65  l2last0daymonthtrickws         float32       \n",
      " 66  l2last0daymonthtrickwus        float32       \n",
      " 67  l2last0daytrickws              float32       \n",
      " 68  l2last0daytrickwus             float32       \n",
      " 69  l2last0womtrickws              int32         \n",
      " 70  l2last0womtrickwus             float32       \n",
      " 71  l1last0daymonthtrickws         float32       \n",
      " 72  l1last0daymonthtrickwus        float32       \n",
      " 73  l1last0daytrickws              float32       \n",
      " 74  l1last0daytrickwus             float32       \n",
      " 75  l1last0womtrickws              float32       \n",
      " 76  l1last0womtrickwus             float32       \n",
      " 77  user_idjdsum                   float32       \n",
      " 78  user_idjdsize                  float32       \n",
      " 79  user_idjdrate                  float32       \n",
      " 80  user_idjlatesum                float32       \n",
      " 81  user_idjlatesize               float32       \n",
      " 82  user_idjlaterate               float32       \n",
      " 83  user_idjtooearlysum            float32       \n",
      " 84  user_idjtooearlysize           float32       \n",
      " 85  user_idjtooearlyrate           float32       \n",
      " 86  user_idjlast0daymonthtrickws   float32       \n",
      " 87  user_idjlast0daymonthtrickwus  float32       \n",
      " 88  user_idjlast0daytrickws        float32       \n",
      " 89  user_idjlast0daytrickwus       float32       \n",
      " 90  user_idjlast0womtrickws        float32       \n",
      " 91  user_idjlast0womtrickwus       float32       \n",
      "dtypes: bool(3), datetime64[ns](6), float32(60), int32(16), object(7)\n",
      "memory usage: 1.9+ GB\n",
      "None\n"
     ]
    }
   ],
   "execution_count": 43
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:57:24.499824Z",
     "start_time": "2025-06-09T10:56:29.732473Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 生成第一期城市相关特征\n",
    "df = repayf(df, r, 'id_city', 'j', 0)\n",
    "# 生成第一期还款天数相关特征\n",
    "df = repayf(df, r, 'l2', 'j', 1)\n",
    "df = repayf(df, r, 'dd', 'j', 0)\n",
    "df = repayf(df, r, 'l1', 'j', 1)\n",
    "df = repayf(df, r, 'im', 'j', 1)\n",
    "df = repayf(df, r, 'eweek', 'j', 1)\n",
    "# 生成 第一期利率和期限 相关特征\n",
    "df = repayf(df, r, 'rate', 'j', 0)\n",
    "df = repayf(df, r, 'term', 'j', 0)\n",
    "print(3)  # 进度标记"
   ],
   "id": "54ebc4c556b68aec",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id_city\n",
      "runtime: 9.520584106445312\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 107 entries, user_id to id_cityjlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(75), int32(16), object(7)\n",
      "memory usage: 2.2+ GB\n",
      "None\n",
      "l2\n",
      "runtime: 4.140463352203369\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 113 entries, user_id to l2jlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(81), int32(16), object(7)\n",
      "memory usage: 2.3+ GB\n",
      "None\n",
      "dd\n",
      "runtime: 7.046108961105347\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 128 entries, user_id to ddjlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(89), int32(23), object(7)\n",
      "memory usage: 2.5+ GB\n",
      "None\n",
      "l1\n",
      "runtime: 4.3963398933410645\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 134 entries, user_id to l1jlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(95), int32(23), object(7)\n",
      "memory usage: 2.7+ GB\n",
      "None\n",
      "im\n",
      "runtime: 4.258477449417114\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 140 entries, user_id to imjlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(100), int32(24), object(7)\n",
      "memory usage: 2.8+ GB\n",
      "None\n",
      "eweek\n",
      "runtime: 4.579878807067871\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 146 entries, user_id to eweekjlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(105), int32(25), object(7)\n",
      "memory usage: 2.9+ GB\n",
      "None\n",
      "rate\n",
      "runtime: 8.876697301864624\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 161 entries, user_id to ratejlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(120), int32(25), object(7)\n",
      "memory usage: 3.2+ GB\n",
      "None\n",
      "term\n",
      "runtime: 8.271954536437988\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4928396 entries, 0 to 4928395\n",
      "Columns: 176 entries, user_id to termjlast0womtrickwus\n",
      "dtypes: bool(3), datetime64[ns](6), float32(128), int32(32), object(7)\n",
      "memory usage: 3.4+ GB\n",
      "None\n",
      "3\n"
     ]
    }
   ],
   "execution_count": 44
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:57:31.181874Z",
     "start_time": "2025-06-09T10:57:28.040628Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 还款日志基础统计特征\n",
    "# 是否还款\n",
    "repay_log_df['repay'] = repay_log_df['repay_date'].astype('str').apply(lambda x: 1 if x != '2200-01-01' else 0)  \n",
    "# 提前还款天数\n",
    "repay_log_df['early_repay_days'] = (repay_log_df['due_date'] - repay_log_df['repay_date']).dt.days  \n",
    "# 处理逾期\n",
    "repay_log_df['early_repay_days'] = repay_log_df['early_repay_days'].apply(lambda x: x if x >= 0 else -1) "
   ],
   "id": "640bf59fee2d12c6",
   "outputs": [],
   "execution_count": 45
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:57:31.997790Z",
     "start_time": "2025-06-09T10:57:31.989962Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 删除不需要的列\n",
    "for f in ['listing_id', 'order_id', 'due_date', 'repay_date', 'auditing_date', 'repay_amt']:\n",
    "    del repay_log_df[f]"
   ],
   "id": "bfefea15dd8f5640",
   "outputs": [],
   "execution_count": 46
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:57:34.885992Z",
     "start_time": "2025-06-09T10:57:33.591729Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 按用户ID分组计算统计量\n",
    "group = repay_log_df.groupby('user_id', as_index=False)\n",
    "# 还款行为统计\n",
    "repay_log_df = repay_log_df.merge(\n",
    "    group['repay'].agg({'repay_mean': 'mean', 'repay_usize': 'size'}), on='user_id', how='left'\n",
    ")"
   ],
   "id": "2c15676804a36683",
   "outputs": [],
   "execution_count": 47
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T10:57:44.134994Z",
     "start_time": "2025-06-09T10:57:42.686279Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 提前还款天数统计\n",
    "repay_log_df = repay_log_df.merge(\n",
    "    group['early_repay_days'].agg({\n",
    "        'early_repay_days_max': 'max', 'early_repay_days_median': 'median', 'early_repay_days_sum': 'sum',\n",
    "        'early_repay_days_n': 'nunique',\n",
    "        'early_repay_days_mean': 'mean', 'early_repay_days_std': 'std'\n",
    "    }), on='user_id', how='left'\n",
    ")"
   ],
   "id": "8eb7cf4a2b4408d0",
   "outputs": [],
   "execution_count": 48
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:04:12.566707Z",
     "start_time": "2025-06-09T10:57:55.423225Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 应还金额统计\n",
    "repay_log_df = repay_log_df.merge(\n",
    "    group['due_amt'].agg({\n",
    "        'due_amt_max': 'max', 'due_amt_min': 'min', 'due_amt_median': 'median',\n",
    "        'due_amt_mean': 'mean', 'due_amt_sum': 'sum', 'due_amt_std': 'std',\n",
    "        'due_amt_skew': 'skew', 'due_amt_kurt': kurtosis, 'due_amt_ptp': np.ptp\n",
    "    }), on='user_id', how='left'\n",
    ")"
   ],
   "id": "6d3265ed0b670b84",
   "outputs": [],
   "execution_count": 49
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:04:19.741654Z",
     "start_time": "2025-06-09T11:04:19.732738Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 清理临时列\n",
    "for i in rpf:\n",
    "    repay_log_df.pop(i)\n",
    "del repay_log_df['repay'], repay_log_df['early_repay_days'], repay_log_df['due_amt']"
   ],
   "id": "d2f01788ad349f32",
   "outputs": [],
   "execution_count": 50
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:04:21.737581Z",
     "start_time": "2025-06-09T11:04:21.512010Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 去重\n",
    "repay_log_df = repay_log_df.drop_duplicates('user_id').reset_index(drop=True)\n",
    "# 删除主数据中已存在的列\n",
    "for i in repay_log_df.copy():\n",
    "    if i in df.keys() and i != 'user_id':\n",
    "        repay_log_df.pop(i)"
   ],
   "id": "5ba009a7b4b943b2",
   "outputs": [],
   "execution_count": 51
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:04:27.616404Z",
     "start_time": "2025-06-09T11:04:22.826448Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 合并到主数据\n",
    "df = df.merge(repay_log_df, on='user_id', how='left')"
   ],
   "id": "2c8ae5dfd6d50830",
   "outputs": [],
   "execution_count": 52
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:04:52.859582Z",
     "start_time": "2025-06-09T11:04:28.253925Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 处理用户行为日志数据\n",
    "user_bh = pd.read_csv('data/user_behavior_logs.csv')"
   ],
   "id": "ef3cd7337365f524",
   "outputs": [],
   "execution_count": 53
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:08.030327Z",
     "start_time": "2025-06-09T11:04:59.431937Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 计算用户行为统计量\n",
    "g = user_bh.groupby(['user_id'])['behavior_type'].agg(\n",
    "    bhsize='size',\n",
    "    bhuni='nunique'\n",
    ").reset_index()\n",
    "# 计算每种行为类型的计数\n",
    "gu = user_bh.groupby(['user_id', 'behavior_type'])['behavior_time'].size().to_dict()\n",
    "# 合并到主数据\n",
    "df = df.merge(g, on='user_id', how='left')"
   ],
   "id": "d4ff6eaa0f4b89ba",
   "outputs": [],
   "execution_count": 54
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:19.179610Z",
     "start_time": "2025-06-09T11:05:13.389208Z"
    }
   },
   "cell_type": "code",
   "source": [
    "bh = []\n",
    "# 计算每种行为类型的详细统计量\n",
    "for i in range(1, 4):\n",
    "    # 每种行为类型的计数\n",
    "    df['bh' + str(i) + 'count'] = df['user_id'].apply(lambda x: gu.get((x, i), 0))  \n",
    "    # 是否有该行为\n",
    "    df['bhtype' + str(i)] = df['bh' + str(i) + 'count'] > 0  \n",
    "    bh.append('bh' + str(i) + 'count')\n",
    "# 计算行为总数和各行为占比\n",
    "df['bhsum'] = df[bh].sum(axis=1)\n",
    "for i in range(1, 4):\n",
    "    df['bh' + str(i) + 'perc'] = df['bh' + str(i) + 'count'] / df['bhsum']"
   ],
   "id": "52f12d397bd4cf7d",
   "outputs": [],
   "execution_count": 55
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:19.940519Z",
     "start_time": "2025-06-09T11:05:19.935239Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 处理重复列名\n",
    "for i in list(df.keys().copy()):\n",
    "    if '_y' in i:\n",
    "        df.pop(i)\n",
    "        df[i[:-2]] = df[i[:-1] + 'x']\n",
    "        df.pop(i[:-1] + 'x')"
   ],
   "id": "20a06100cf92f12",
   "outputs": [],
   "execution_count": 56
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:25.692210Z",
     "start_time": "2025-06-09T11:05:21.067127Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 创建城市-年龄-还款天数组合特征\n",
    "df['city_age_l1'] = df['id_city'].astype(str) + df['age_bin'].astype(str) + df['l1'].astype(str)\n",
    "# 计算组合特征的平均金额\n",
    "city_age_bin_mapping = df.groupby('city_age_l1').agg({'principal': 'mean', 'due_amt': 'mean'})\n",
    "# 添加相对于平均值的比率\n",
    "df['mean_principal_by_city_age_l1'] = df['city_age_l1'].map(city_age_bin_mapping['principal'])\n",
    "df['principal_over_by_city_age_l1'] = df['principal'] / df['mean_principal_by_city_age_l1']\n",
    "df['mean_due_amt_by_city_age_l1'] = df['city_age_l1'].map(city_age_bin_mapping['due_amt'])\n",
    "df['due_amt_over_by_city_age_l1'] = df['due_amt'] / df['mean_due_amt_by_city_age_l1']\n",
    "# 删除临时列\n",
    "df.drop(['mean_principal_by_city_age_l1', 'mean_due_amt_by_city_age_l1'], inplace=True, axis=1)"
   ],
   "id": "5c544405879ef4b6",
   "outputs": [],
   "execution_count": 57
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:26.876275Z",
     "start_time": "2025-06-09T11:05:26.656869Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 计算金额与天数的比率特征\n",
    "df['due_amt_per_days'] = df['due_amt'] / (df['due_date'] - df['auditing_date']).dt.days\n",
    "df['due_amt_per_sdays'] = df['due_amt'] / np.sqrt((df['due_date'] - df['auditing_date']).dt.days)\n",
    "w = df['due_amt_per_days'].copy()"
   ],
   "id": "82bfa1650d99ae9",
   "outputs": [],
   "execution_count": 58
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:31.934935Z",
     "start_time": "2025-06-09T11:05:28.700059Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 处理日期特征\n",
    "date_cols = ['auditing_date', 'due_date', 'reg_mon', 'info_insert_date', 'tag_insert_date']\n",
    "for f in date_cols:\n",
    "    if f in ['reg_mon', 'info_insert_date', 'tag_insert_date']:\n",
    "        df[f + '_year'] = df[f].dt.year  # 年份\n",
    "\n",
    "    df[f + '_month'] = df[f].dt.month  # 月份\n",
    "    if f in ['auditing_date', 'due_date', 'info_insert_date', 'tag_insert_date']:\n",
    "        df[f + '_day'] = df[f].dt.day  # 日\n",
    "        df[f + '_dayofweek'] = df[f].dt.dayofweek  # 星期几\n",
    "# 删除原始日期列\n",
    "df.drop(columns=date_cols, axis=1, inplace=True)"
   ],
   "id": "10e5b453f6beb34d",
   "outputs": [],
   "execution_count": 59
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:33.318558Z",
     "start_time": "2025-06-09T11:05:33.179515Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 删除不需要的列\n",
    "del df['user_id'], df['listing_id']\n",
    "print('runtime: {}\\n'.format(time.time() - t))\n",
    "# 标记缺失标签\n",
    "df['nantag'] = df['taglist'].isnull()\n",
    "print(4)  # 进度标记"
   ],
   "id": "c277ebb945c8fcf",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "runtime: 804.4095942974091\n",
      "\n",
      "4\n"
     ]
    }
   ],
   "execution_count": 60
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:37.855704Z",
     "start_time": "2025-06-09T11:05:35.947257Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 类别特征编码\n",
    "cate_cols = ['gender', 'cell_province', 'id_province', 'id_city', 'age_bin', 'city_age']\n",
    "for f in cate_cols:\n",
    "    # 使用factorize(因子化或分类编码)进行高效编码\n",
    "    df[f] = pd.factorize(df[f])[0].astype('int32')\n",
    "\n",
    "for f in ['city_age_l1']:\n",
    "    if f in df.columns:  # 安全检查\n",
    "        df[f] = pd.factorize(df[f])[0].astype('int32')"
   ],
   "id": "9de83b4ccee3ac3d",
   "outputs": [],
   "execution_count": 61
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:38.741699Z",
     "start_time": "2025-06-09T11:05:38.642932Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 删除不需要的列\n",
    "del df['repay_date']\n",
    "del df['taglist']"
   ],
   "id": "198496057cb6ff12",
   "outputs": [],
   "execution_count": 62
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:42.032755Z",
     "start_time": "2025-06-09T11:05:40.898535Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 分批转换数据类型以减少内存峰值\n",
    "num_cols = df.select_dtypes(include=['float64', 'int64']).columns\n",
    "for col in num_cols:\n",
    "    if df[col].dtype == 'float64':\n",
    "        df[col] = df[col].astype('float32')\n",
    "    elif df[col].dtype == 'int64':\n",
    "        # 根据数值范围选择更小的整数类型\n",
    "        max_val = df[col].max()\n",
    "        min_val = df[col].min()\n",
    "\n",
    "        if min_val >= 0:  # 无符号整数\n",
    "            if max_val < 256:\n",
    "                df[col] = df[col].astype('uint8')\n",
    "            elif max_val < 65536:\n",
    "                df[col] = df[col].astype('uint16')\n",
    "            else:\n",
    "                df[col] = df[col].astype('uint32')\n",
    "        else:  # 有符号整数\n",
    "            if min_val >= -128 and max_val < 128:\n",
    "                df[col] = df[col].astype('int8')\n",
    "            elif min_val >= -32768 and max_val < 32768:\n",
    "                df[col] = df[col].astype('int16')\n",
    "            else:\n",
    "                df[col] = df[col].astype('int32')"
   ],
   "id": "fa869ffa2c29bf09",
   "outputs": [],
   "execution_count": 63
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:43.008768Z",
     "start_time": "2025-06-09T11:05:43.004607Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 分批保存到磁盘以避免内存溢出\n",
    "output_file = 'result_data/litedft6.h5'\n",
    "# 每次处理1百万行\n",
    "chunk_size = 1_000_000  "
   ],
   "id": "5609324b17c9c048",
   "outputs": [],
   "execution_count": 64
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:54.634126Z",
     "start_time": "2025-06-09T11:05:44.401098Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 删除现有文件(如果存在)\n",
    "import os\n",
    "if os.path.exists(output_file):\n",
    "    os.remove(output_file)\n",
    "# 分批保存\n",
    "for i in range(0, len(df), chunk_size):\n",
    "    chunk = df.iloc[i:i + chunk_size]\n",
    "    if i == 0:\n",
    "        chunk.to_hdf(output_file, key='df', mode='w', format='table')\n",
    "    else:\n",
    "        chunk.to_hdf(output_file, key='df', mode='a', append=True, format='table')\n",
    "    print(f\"已保存行: {min(i + chunk_size, len(df))}/{len(df)}\")\n",
    "\n",
    "print(f\"成功保存到 {output_file}，总行数: {len(df)}\")"
   ],
   "id": "5d37f8bc42f1e2b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已保存行: 1000000/4928396\n",
      "已保存行: 2000000/4928396\n",
      "已保存行: 3000000/4928396\n",
      "已保存行: 4000000/4928396\n",
      "已保存行: 4928396/4928396\n",
      "成功保存到 litedft6.h5，总行数: 4928396\n"
     ]
    }
   ],
   "execution_count": 65
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:05:56.309419Z",
     "start_time": "2025-06-09T11:05:56.304864Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义平均数编码函数\n",
    "def get_stratifiedkfold_ids(x, y, n_folds=5, random_state=42, shuffle=True):\n",
    "    \"\"\"获取分层K折交叉验证的索引\"\"\"\n",
    "    kfold = StratifiedKFold(n_splits=n_folds, random_state=random_state, shuffle=shuffle)\n",
    "    fold = kfold.split(x, y)\n",
    "    fold_ids = []\n",
    "    for k, (train_in, test_in) in enumerate(fold):\n",
    "        fold_ids.append([train_in, test_in])\n",
    "    return fold_ids"
   ],
   "id": "14497bd3903ac1",
   "outputs": [],
   "execution_count": 66
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:06:00.694603Z",
     "start_time": "2025-06-09T11:06:00.686714Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义均值编码函数，用于特征工程\n",
    "def mean_encoding_feature_label(train, test, col_encode, col_label, alpha=10):\n",
    "    # 对两种类型的特征进行处理（l1和l2）\n",
    "    for k in ['l1', 'l2']:\n",
    "        # 只对标签为1的情况进行处理\n",
    "        for label in [1]:\n",
    "\n",
    "            # 创建新列名\n",
    "            new_col_name = col_encode + k + '_' + str(label) + '_mean_encoding'\n",
    "\n",
    "            # 在训练集和测试集中创建组合特征列\n",
    "            train[col_encode + '_' + k] = train[col_encode].astype(str) + train[k].astype(str)\n",
    "            test[col_encode + '_' + k] = test[col_encode].astype(str) + test[k].astype(str)\n",
    "\n",
    "            # 计算全局目标均值\n",
    "            target_global_mean = train[train[col_label] == label][col_label].count() / train[col_label].count()\n",
    "            # cat_count = train[train[col_label]==label].groupby(col_encode+k)[col_encode+k].count()\n",
    "\n",
    "            # 计算分子和分母\n",
    "            fenzi = train[train[col_label] == label].groupby(col_encode + '_' + k).size()\n",
    "            fenmu = train.groupby(col_encode + '_' + k).size()\n",
    "\n",
    "            # 计算均值编码映射（使用平滑处理）\n",
    "            mean_encoding_mapping = (fenzi / fenmu * fenzi + target_global_mean * alpha) / (fenzi + alpha)\n",
    "            # 将编码应用到测试集\n",
    "            test.loc[:, new_col_name] = test.loc[:, col_encode + '_' + k].map(mean_encoding_mapping)\n",
    "            # 输出\n",
    "            print('Encoding feature: ', col_encode, ' of label: ', label, end=', ')\n",
    "\n",
    "            # 使用分层K折交叉验证防止数据泄露\n",
    "            fold_ids = get_stratifiedkfold_ids(train[col_label], train[col_label],n_folds=5, random_state=2019, shuffle=True)\n",
    "            # 对每一折进行处理\n",
    "            for i, (trainid, validid) in enumerate(fold_ids):\n",
    "                print(' fold :', i, end=' ... ')\n",
    "                trainfold = train.iloc[trainid, :]\n",
    "                # 计算当前折的全局目标均值\n",
    "                target_global_mean = trainfold[trainfold[col_label] == label][col_label].count() / trainfold[col_label].count()\n",
    "                # cat_count = trainfold[trainfold[col_label]==label].groupby(col_encode+k)[col_encode+k].count()\n",
    "\n",
    "                # 计算当前折的分子和分母\n",
    "                fenzi = trainfold[trainfold[col_label] == label].groupby(col_encode + '_' + k).size()\n",
    "                fenmu = trainfold.groupby(col_encode + '_' + k).size()\n",
    "                # 计算当前折的均值编码映射\n",
    "                mean_encoding_mapping = (fenzi / fenmu * fenzi + target_global_mean * alpha) / (fenzi + alpha)\n",
    "                # 将编码应用到验证集\n",
    "                train.loc[validid, new_col_name] = train.loc[validid, col_encode + '_' + k].map(mean_encoding_mapping)\n",
    "                gc.collect()\n",
    "\n",
    "            # 删除临时列\n",
    "            train.pop(col_encode + '_' + k)\n",
    "            test.pop(col_encode + '_' + k)\n",
    "\n",
    "            # 转换数据类型为float32以节省内存\n",
    "            train = train.astype('float32')\n",
    "            test = test.astype('float32')\n",
    "            gc.collect()\n",
    "    return train, test"
   ],
   "id": "9098c6f8c6053aed",
   "outputs": [],
   "execution_count": 67
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:06:03.457226Z",
     "start_time": "2025-06-09T11:06:03.450107Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 分割数据集为训练集和测试集\n",
    "train_values, test_values = df[:train_num], df[train_num:]\n",
    "train_values['label'] = clf_labels"
   ],
   "id": "fe7efdafb1c4ec63",
   "outputs": [],
   "execution_count": 68
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:06:05.151484Z",
     "start_time": "2025-06-09T11:06:05.145921Z"
    }
   },
   "cell_type": "code",
   "source": "train_num",
   "id": "10e69f286848b1f9",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "811318"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 69
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:07:28.672384Z",
     "start_time": "2025-06-09T11:07:28.656996Z"
    }
   },
   "cell_type": "code",
   "source": [
    "train_num_df = pd.DataFrame({'number':[train_num]})\n",
    "train_num_df"
   ],
   "id": "876bc10a999c8a06",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   number\n",
       "0  811318"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>811318</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 70
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:08:35.574921Z",
     "start_time": "2025-06-09T11:08:35.558791Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 保存为 Parquet（推荐）\n",
    "train_num_df.to_parquet(\"result_data/train_num_df.parquet\")"
   ],
   "id": "46e928e26d16b71b",
   "outputs": [],
   "execution_count": 72
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:12:31.640504Z",
     "start_time": "2025-06-09T11:10:54.534086Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 第一组特征进行均值编码\n",
    "cl = ['gender', 'id_province', 'cell_province', 'id_city', 'city_age']\n",
    "for i in cl:\n",
    "    train_values, test_values = mean_encoding_feature_label(train_values, test_values, i, 'label', alpha=10)\n",
    "    print(train_values.shape, test_values.shape)"
   ],
   "id": "3cb025a74f814b04",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding feature:  gender  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  gender  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 221) (4117078, 220)\n",
      "Encoding feature:  id_province  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  id_province  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 223) (4117078, 222)\n",
      "Encoding feature:  cell_province  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  cell_province  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 225) (4117078, 224)\n",
      "Encoding feature:  id_city  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  id_city  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 227) (4117078, 226)\n",
      "Encoding feature:  city_age  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  city_age  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 228) (4117078, 227)\n"
     ]
    }
   ],
   "execution_count": 76
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:13:58.359683Z",
     "start_time": "2025-06-09T11:12:41.040583Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 第二组特征进行均值编码\n",
    "cl = ['term', 'rate', 'bhtype3', 'nantag']\n",
    "for i in cl:\n",
    "    train_values, test_values = mean_encoding_feature_label(train_values, test_values, i, 'label',alpha=10)\n",
    "    print(train_values.shape, test_values.shape)"
   ],
   "id": "28e581199dc36614",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding feature:  term  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  term  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 230) (4117078, 229)\n",
      "Encoding feature:  rate  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  rate  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 232) (4117078, 231)\n",
      "Encoding feature:  bhtype3  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  bhtype3  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 234) (4117078, 233)\n",
      "Encoding feature:  nantag  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  nantag  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 236) (4117078, 235)\n"
     ]
    }
   ],
   "execution_count": 77
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:14:21.721810Z",
     "start_time": "2025-06-09T11:14:03.501926Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 第三组特征进行均值编码\n",
    "cl = ['last0weekofmonth']\n",
    "for i in cl:\n",
    "    train_values, test_values = mean_encoding_feature_label(train_values, test_values, i, 'label', alpha=10)\n",
    "    print(train_values.shape, test_values.shape)"
   ],
   "id": "771e88b888e8a700",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding feature:  last0weekofmonth  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... Encoding feature:  last0weekofmonth  of label:  1,  fold : 0 ...  fold : 1 ...  fold : 2 ...  fold : 3 ...  fold : 4 ... (811318, 238) (4117078, 237)\n"
     ]
    }
   ],
   "execution_count": 78
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:14:27.137231Z",
     "start_time": "2025-06-09T11:14:26.121467Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 删除标签列\n",
    "train_values.pop('label')\n",
    "# 转换数据类型为float32以节省内存\n",
    "train_values = train_values.astype('float32')\n",
    "test_values = test_values.astype('float32')"
   ],
   "id": "8f9d4ef12859fd58",
   "outputs": [],
   "execution_count": 79
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-06-09T11:14:36.918635Z",
     "start_time": "2025-06-09T11:14:28.461072Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 保存处理后的数据\n",
    "train_values.to_hdf('result_data/litetrain6' + '.h5', key='df', mode='w')\n",
    "test_values.to_hdf('result_data/litetest6' + '.h5', key='df', mode='w')"
   ],
   "id": "c812a8f121024d84",
   "outputs": [],
   "execution_count": 80
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
